package common.main.newsWangyiSmart;

import com.google.gson.Gson;
import common.http.HtmlInfo;
import common.http.SimpleHttp;
import common.utils.DomTree;
import common.utils.MD5Util;
import org.apache.log4j.Logger;
import org.w3c.dom.DocumentFragment;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;

public class NewsWangyiSmartMain {
    static Logger logger = Logger.getLogger(NewsWangyiSmartMain.class);
    static SimpleHttp http = new SimpleHttp();
    static Gson gson = new Gson();

    static String ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36";
    static String gb2312 = "gb2312";
    static String utf8 = "utf-8";

    public static void main(String[] args) throws InterruptedException {
        //获取MD5防止重复
        String allMd5 = NewsWangyiSmartSql.getAllMd5();
        HtmlInfo html = new HtmlInfo();
        html.setUa(ua);
        html.setEncode(gb2312);

        String modelUrl = "http://tech.163.com/special/00097UHL/smart_datalist<page>.js?callback=data_callback";
        String[]page = {"","_02","_03","_04","_05","_06","_07","_08","_09","_10","_11","_12","_13","_14","_15","_16","_17","_18","_19","_20",};
        while (true){
            //获取列表数据
            for(int i = 0;i < 20;i++){
                logger.info("15s后开始获取第"+(i + 1)+"页的列表数据");
                Thread.sleep(15 * 1000);
                html.setOrignUrl(modelUrl.replace("<page>",page[i]));
                http.simpleGet(html);

                String json = html.getContent().substring(0,html.getContent().length() - 1).replace("data_callback(","");
                List<Map> list = gson.fromJson(json,List.class);

                List<NewsWangyiSmartEntity> entityList = new ArrayList<>();

                for(int j = 0;j < list.size();j++){
                    Map newsMap = list.get(j);
                    NewsWangyiSmartEntity entity = new NewsWangyiSmartEntity();

                    entity.setTitle((String) newsMap.get("title"));
                    entity.setPubTime((String) newsMap.get("time"));
                    entity.setUrl((String) newsMap.get("docurl"));
                    entity.setMd5(MD5Util.MD5(entity.getUrl()));
                    logger.info("15s后开始获取《"+ entity.getTitle() +"》的详细信息");
                    if(allMd5.contains(entity.getMd5())){
                        logger.info("该新闻已存在数据库中，跳过采集");
                        continue;
                    }
                    allMd5 = allMd5 + entity.getMd5() + ",";
                    Thread.sleep(15 * 1000);
                    entity.setSource((String) newsMap.get("label"));

                    List<Map> keywordList = (List<Map>) newsMap.get("keywords");
                    if(keywordList != null){
                        String keywords = "";
                        for(int a = 0;a < keywordList.size();a++){
                            keywords = keywords + keywordList.get(a).get("keyname") + ",";
                        }
                        entity.setLabel(keywords);
                    }
                    html.setOrignUrl(entity.getUrl());
                    http.simpleGet(html);

                    DocumentFragment detailPageNode = DomTree.getNode(html.getContent(),html.getEncode());

                    //获取新闻的摘要
                    NodeList briefNodeList = DomTree.commonList(NewsWangyiSmartXpath.briefXpath,detailPageNode);
                    if(briefNodeList != null && briefNodeList.getLength() > 0){
                        entity.setBrief(briefNodeList.item(0).getTextContent());
                    }
                    //获取新闻带有Html标签的内容
                    NodeList htmlContentNodeList = DomTree.commonList(NewsWangyiSmartXpath.htmlContentXpath,detailPageNode);
                    if(htmlContentNodeList != null && htmlContentNodeList.getLength() > 0){
                        NodeList nodeList = htmlContentNodeList.item(0).getChildNodes();
                        String htmlContent = getContentWithHtml(nodeList);
                        entity.setHtmlContent(htmlContent);
                    }
                    //获取新闻的内容
                    NodeList contentNodeList = DomTree.commonList(NewsWangyiSmartXpath.contentXpath,detailPageNode);
                    if(contentNodeList != null){
                        String content = "";
                        for(int a = 0;a < contentNodeList.getLength();a++){
                            content = content + contentNodeList.item(a).getTextContent() + "\n";
                        }
                        entity.setContent(content);
                    }
                    //获取新闻图片Url
                    NodeList imgUrlNodeList = DomTree.commonList(NewsWangyiSmartXpath.imgUrlXpath,detailPageNode);
                    if(imgUrlNodeList != null ){
                        String imgUrl = "";
                        for(int a = 0;a < imgUrlNodeList.getLength();a++){
                            imgUrl = imgUrl + imgUrlNodeList.item(a).getTextContent() + ",";
                        }
                        entity.setImgUrl(imgUrl);
                    }

                    if(entity.getContent() != null && !"".equals(entity.getContent())){
                        entityList.add(entity);
                    }else {
                        logger.info(entity.getTitle() + "的内容为空，跳过采集");
                    }
                }
                //采集完一页之后保存到数据库。
                logger.info("该页数据采集完成，保存到数据库");
                NewsWangyiSmartSql.saveData(entityList);
                logger.info("数据保存完成，size=" + entityList.size());
            }
            logger.info("数据采集完成，等待1小时后继续采集");
            Thread.sleep(60 * 60 * 1000);
        }
    }

    //获取页面Html
    private static String getContentWithHtml(NodeList nodeList){
        String textWithHtml = "";
        for(int i = 0;i<nodeList.getLength();i++){
            Node node = nodeList.item(i);
            String nodeName = node.getNodeName().toLowerCase();
            String imgUrl = null;
            String childContentWithHtml = null;
            String nodeText = null;
            if(node.hasChildNodes()){
                NodeList childNodeList = node.getChildNodes();
                childContentWithHtml = getContentWithHtml(childNodeList);
                textWithHtml = textWithHtml + "\n" + "<" + nodeName + ">" + childContentWithHtml + "</" + nodeName + ">";
            }else {
                if("img".equals(nodeName)){
                    try{
                        imgUrl = node.getAttributes().getNamedItem("src").getTextContent();
                    }catch (NullPointerException e){
                        imgUrl = node.getAttributes().getNamedItem("srcset").getTextContent().split(",")[0].split(" ")[0];
                    }
                    textWithHtml = textWithHtml + "\n" + "<img src=\"" + imgUrl + "\"" + "</img>";
                }else {
                    nodeText = node.getTextContent();
                    textWithHtml = textWithHtml + "\n" + nodeText;
                }
            }
        }
        return textWithHtml;
    }

}
